{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 载入数据"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "from urllib import request"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "target_url = (\"https://archive.ics.uci.edu/ml/machine-learning-\"\n",
    "\"databases/undocumented/connectionist-bench/sonar/sonar.all-data\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "req = request.Request(target_url, headers={\"User-Agent\": \"Mozilla/5.0\"})\n",
    "with request.urlopen(req) as resp:\n",
    "    data = resp.read()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "x_list = []\n",
    "labels = []\n",
    "\n",
    "# 注意，这样读进来的数据都是字符串\n",
    "for line in data.decode('utf-8').split('\\n'):\n",
    "    if not line.strip():\n",
    "        continue\n",
    "    \n",
    "    row = line.strip().split(',')\n",
    "    x_list.append(row)\n",
    "    labels.append(row[-1])\n",
    "    \n",
    "nrows = len(x_list)\n",
    "ncols = len(x_list[1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of Rows:  208\n",
      "Number of Columns:  61\n"
     ]
    }
   ],
   "source": [
    "print(\"Number of Rows: \", nrows)\n",
    "print(\"Number of Columns: \", ncols)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 属性统计信息， 目前对应的是：**数值型数据**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### column 3的基础统计信息"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "col = 3\n",
    "col_data = []\n",
    "\n",
    "for row in x_list:\n",
    "    col_data.append(float(row[col]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Column 3的基础统计信息： Mean = 0.053892307692307684  Standard Deviation = 0.04641598322260027\n"
     ]
    }
   ],
   "source": [
    "col_arr = np.array(col_data)\n",
    "col_mean = np.mean(col_arr)\n",
    "col_std = np.std(col_arr)\n",
    "\n",
    "print(\"Column 3的基础统计信息：\", \"Mean = \" + str(col_mean) + '  ' + \"Standard Deviation = \" + str(col_std))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### column 3的分位点统计"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Boundaries for 4 Equal Percentiles: \n",
      "[0.0058, 0.024375, 0.04405, 0.0645, 0.4264]\n"
     ]
    }
   ],
   "source": [
    "ntiles = 4\n",
    "pct_boundary = []\n",
    "for i in range(ntiles+1):\n",
    "    # [0,4]\n",
    "    pct_boundary.append(np.percentile(col_arr, int((i/ntiles)*100)))\n",
    "    \n",
    "print(\"Boundaries for 4 Equal Percentiles: \")\n",
    "print(pct_boundary)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Boundaries for 10 Equal Percentiles: \n",
      "[0.0058, 0.0141, 0.022740000000000003, 0.027869999999999995, 0.03622, 0.04405, 0.05071999999999999, 0.059959999999999986, 0.07794000000000001, 0.10836, 0.4264]\n"
     ]
    }
   ],
   "source": [
    "ntiles = 10\n",
    "pct_boundary = []\n",
    "for i in range(ntiles+1):\n",
    "    # [0,10]\n",
    "    pct_boundary.append(np.percentile(col_arr, int((i/ntiles)*100)))\n",
    "    \n",
    "print(\"Boundaries for 10 Equal Percentiles: \")\n",
    "print(pct_boundary)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "***从最后两个数据的间隔看出，尾部数据开始变的稀疏，可能存在极端值***。不过有些时候尾部稀疏是正常的，如卡方分布在尾部常会变得稀疏。所以要具体分析"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 标签出具统计，目前对应的是：**类别数据（categorical variables）**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Unique Label Values: \n",
      "{'M', 'R'}\n"
     ]
    }
   ],
   "source": [
    "#The last column contains categorical variables\n",
    "col = 60\n",
    "col_data = []\n",
    "\n",
    "for row in x_list:\n",
    "    col_data.append(row[col])\n",
    "\n",
    "unique_cat = set(col_data)\n",
    "print(\"Unique Label Values: \")\n",
    "print(unique_cat)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 对类别数据数量进行统计"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "#count up the number of elements having each value\n",
    "cat_dict = dict.fromkeys(unique_cat, 0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "for cat in col_data:\n",
    "    cat_dict[cat] = cat_dict.get(cat, 0) + 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Counts for Each Value of Categorical Label: \n",
      "{'M': 111, 'R': 97}\n"
     ]
    }
   ],
   "source": [
    "print(\"Counts for Each Value of Categorical Label: \")\n",
    "print(cat_dict)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 其实可以使用collections.Counter进行"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "from collections import Counter"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Counter({'R': 97, 'M': 111})"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "obj_cat = Counter(col_data)\n",
    "obj_cat"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('M', 111)]"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "obj_cat.most_common(1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
